'''
Created on Feb 19, 2011

@author: solagna
'''
import BeautifulSoup.BeautifulSoup

def isStub (stubStr):
    '''
        Check if a string (most probably parsed from a wiki page) is a stub or it has been edited by a user.
        Heuristic algorithm: a string is a stub if:
        - begins and end with < >
        - It is empty
        - It is "Test"
        ...
    '''
    answer = False
    #Stub is a string that starts with '<' and ends with '>'. 
    stubStr = stubStr.replace(' ', '')
    stubStr = stubStr.replace('\n', '')
    if stubStr.endswith('>') and stubStr.startswith('<'):
        answer =  True
    if stubStr.endswith('&gt;') and stubStr.startswith('&lt;'):
        answer =  True
    if stubStr == "":
        answer = True #Empty string is also a stub
    if stubStr == "Test":
        answer = True
        
    return answer


def soupify (filename):
    htmlCode = open (filename, "r").read()
    return BeautifulSoup.BeautifulSoup.BeautifulSoup(htmlCode)

def unicodelist (list_of_strings):
    ''' Returns the same lists with elements converted in unicode'''
    return [unicode(i) for i in list_of_strings]

def findTableX (soup, tableN):
    ''' Returns the soup of the Xth table in the page'''
    return soup.findAll('table')[tableN - 1]

def parseTableX (soup, table_number, add_stubs=False):

    ''' It skips, or tries to skip, rows with only stubs, or empty (if skip_stub).'''    
    out_list = []
    tableX = findTableX(soup, table_number)
    
    rows = tableX.findAll('tr')
    for tr in rows:
        #Parse table rows
        row_list = []
        only_stubs = True #Var used to check if in the row there are only stubs (not edited by nobody).
        cols = tr.findAll('td')
        for td in cols:
            #Parse row columns
            column_raw_text = td.find(text=True)
            column_text = unicode(column_raw_text).rstrip(" \n")
            row_list.append(column_text) #Remove trailing spaces or newline chars, and append.
            only_stubs = only_stubs and isStub(column_text)
        
        if not only_stubs or add_stubs:
            out_list.append(row_list) #Append the row to the output list        
    return out_list


def parseTableYColHeader (table):
    '''
    It parses the header column, if extisting, of the Yth table in the web page. Returns the values in a list.
    Empty list returned if header not found. Header is the first row, if extists.
    No stub-check.
    '''
    return parseTableColHeader (table, 1)

def parseTableColHeader (table,col_num):
    header_list = list()
    for row in table.findAll('tr'):
        th = row.findAll('th')[col_num - 1]
        header_list.append(unicode(th.find(text=True)).rstrip(" \n"))
    return header_list


def parseTableYheader (soup, table):
    '''
    It parses the header row, if extisting, of the Yth table in the web page. Returns the values in a list.
    Empty list returned if header not found. Header is the first row, if extists.
    No stub-check.
    '''
    row_list = list()
    table = findTableX(soup, table)
    header = table.findAll('tr')[0] # <th> is the table header.
    cols = header.findAll('th')
    for th in cols:
        column_text = unicode(th.find(text=True)).rstrip(" \n")
        print column_text + " <<"
        row_list.append(column_text) #Remove trailing spaces or newline chars, and append.
        #only_stubs = only_stubs and isStub(column_text)
        #Parse row columns
    return row_list

    
def parseColumnYfromTableX (soup, column, tableN, stubs_empy=True):
    '''Returns a list containing the elements from the Yth column of the Xth table in the soup page
        Imput : soup-ed http page, y column, x table
        stubs_empty (default TRUE): if true replace stubs like values with an empty string.
        Output: list of elements
        NOTE : COLUMNS START FROM 1
    '''
    out_list = []
    table = parseTableX(soup, tableN, True) #Add also stubs
    for row in table:
        skip_row = False
        try:
            to_append = row[column -1]
        except:
            #The row is empy, most probably. It would be good to skip all the row.
            skip_row = True
        if not skip_row:
            if stubs_empy: #Check or not if the string is stub.
                if isStub(row[column -1]): #It is stub.
                    to_append = ""
            out_list.append(to_append)
        
    return out_list
        